# coding: utf8


import os,sys
sys.path.append('..')

import logging
import time
import re
import traceback
import requests_html
from multiprocessing.pool import ThreadPool
from server.post_data import PostHelper

"""大连理工大学
微电子学院
"""

logging.basicConfig(level=logging.INFO)

session = requests_html.HTMLSession()
pool = ThreadPool(os.cpu_count() * 2)  # 要使用 hanlp, 线程数设为1
host = "http://faculty.dlut.edu.cn"


add_breakline = re.compile(r'(\s+([一二三四五六七])|([123456789][. ]+)\s*\S+)')  # 用于区分主目录
reduce_breakline = re.compile(r"[ \n]+")  # 避免与主目录混淆
reduce_breakline2 = re.compile(r"\n{2,}")
reduce_dollar = re.compile(r'(\$\$\s*){2,}')
reduce_dollar2 = re.compile(r'(\$\s*)+')

avatar_ptn = re.compile(r'u_u5_pic\.addimg\("(\S+?")')

sex_ptn = re.compile(r'([男|女])')
degree_ptn = re.compile(r'学位[:：](\S+)')
dep_ptn = re.compile(r'所在单位[:：](\S+)')
subj_ptn = re.compile(r'学科[:：](\S+)')

addr_ptn = re.compile(r'(?:通信地址|通讯地址|办公地址|办公地点|Address)[:：]([ \S]+)')
tel_ptn = re.compile(r'(?:办公电话|电话|联系方式|Tel|tel)[:：]([0-9+\- ]+)')
email_ptn = re.compile(r'(?:电子邮箱|电子信箱|E-mail|Email|email|e-mail|邮箱地址|邮\s*箱)[:：](\S+@[.\S]+)')
ri_ptn = re.compile(r'研究(?:方向|领域|重点)(?:为|包括)*[:：\n]*([ \n\S]+?[。\n])')
pos_ptn = re.compile(r"(教授|副教授|助理教授|研究员|副研究员|博士导师|硕士导师|博士生研究生|硕士生导师|IEEE Fellow|IEEE\s*会员|博士|讲师|教师|高级工程师|工程师)")
edu_ptn = re.compile(r"(?:考入|攻读|进修|毕业于|学位|博士后|访问|工程师)")
split_ptn = re.compile(r'([;；。\n])')


post_helper = PostHelper()


def teacher_list():
    try:
        r = session.get("http://smedut.dlut.edu.cn/szdw/jsxx.htm")
        if r.status_code == 200:
            ts = r.html.xpath("//div[contains(@class, 'cate_listRow fl')]")
            teachers = []

            for t in ts:
                pos = t.xpath("//h1[1]")[0].full_text
                lis = t.xpath("//ul[1]//li")

                for li in lis:
                    a = li.xpath("//a[1]")[0]
                    url = list(a.absolute_links)[0]
                    name = a.full_text

                    logging.info("name: %s, pos: %s, url: %s", name, pos, url)
                    teachers.append((name, url))

            pool.imap_unordered(teacher_detail, teachers)
            pass
        else:
            logging.warning("wrong status code: %s", r.status_code)
            r.raise_for_status()
    except Exception as e:
        traceback.print_exc()
        raise e


# 该学校的教师信息存在多种模板，这里只选择了其中一种进行抽取。
def teacher_detail(payload):
    name, url = payload
    logging.info("loading %s ...", name)

    try:
        r = session.get(url)
        if r.status_code == 200:
            avatar = r.html.xpath("//div[@class='t_photo'][1]")
            avatar = avatar[0].full_text if avatar else ''
            avatar = avatar_ptn.search(avatar)
            avatar = host + avatar.group(1).strip('"') if avatar else ''

            # name = r.html.xpath("//div[@class='t_photo'][1]//span[1]/text()")
            # name = name[0] if name else ''

            basic_info = r.html.xpath("//div[@class='t_jbxx_nr'][1]")
            basic_info = basic_info[0].full_text if basic_info else ''

            sex, degree, dep, subj, addr, tel, email = [ptn.search(basic_info) for ptn in (sex_ptn, degree_ptn, dep_ptn, subj_ptn, addr_ptn, tel_ptn, email_ptn)]
            sex, degree, dep, subj, addr, tel, email = [(match.group(1).strip() if match else '') for match in (sex, degree, dep, subj, addr, tel, email)]
            pos = ",".join(list(set(pos_ptn.findall(basic_info))))

            intro = r.html.xpath("//div[@class='t_grjj_nr'][1]")
            intro = intro[0].full_text if intro else ''

            sentences = split_ptn.split(intro)
            edu_sent = []
            for sent in sentences:
                if edu_ptn.search(sent):
                    edu_sent.append(sent)
            edu = "。".join(edu_sent)

            ri = r.html.xpath("//ul[@class='shjz-list'][1]/preceding-sibling::ul")
            ri = ri[0].full_text if ri else ''
            ri = ri.replace('\n', ' ')

            logging.info("name: %s, sex: %s, pos: %s, degree: %s, dep: %s, subj: %s, addr: %s, tel: %s, email: %s, edu: %s, ri: %s, avatar: %s", name, sex, pos, degree, dep, subj, addr, tel, email, edu, ri, avatar)

            post_helper.post_expert_data(name, '大连理工大学', dep, pos, degree, tel, email, addr, avatar, edu, '', ri, '', intro, '大连理工大学', url)
            pass
        else:
            logging.warning("wrong status code: %s (name: %s)", r.status_code, name)
            # r.raise_for_status()
    except Exception as e:
        traceback.print_exc()
        raise e


if __name__ == '__main__':
    teacher_list()

    pool.close()
    pool.join()

    session.close()

    post_helper.close()
    pass



